# -*- coding: utf-8 -*-
import re
import time
from lxml import etree
import scrapy
from scrapy.http import Request
from bs4 import BeautifulSoup as bs
import sys

from dingdian.items import SecuniaItem
from dingdian.schedule import schedule

reload(sys)
sys.setdefaultencoding( "utf-8" )

class Myspider(scrapy.Spider):

    name = 'secunia'
    bash_url = 'https://secuniaresearch.flexerasoftware.com/community/research/'

    cvePattern = re.compile(r'CVE-\d{4}-\d+')
    timePattern = re.compile(r'\d+[/,-]\d+[/,-]\d+')
    vulNumPattern = re.compile(r'\d+')
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cache-Control': 'max-age=0',
        'Pragma': 'no-cache',
        'Referer': 'https://www.cvedetails.com/index.php',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'
    }
    pageNum = 0
    vulNum = 0
    totalVuls = 0
    currentVuls = 0.0
    def start_requests(self):
        url = self.bash_url
        yield Request(url,headers=self.headers,callback= self.parse)

    def parse(self, response):
        #print response.text
        table = bs(response.text,'html5lib').find_all('div',attrs={'class':'srlist'})
        for i in range(0,len(table)):
            vulNum = table[i].find('p',attrs={'class':'no-margin'}).get_text()
            self.totalVuls+=int( re.findall(self.vulNumPattern, vulNum)[-1])
            vul = table[i].find_all('tr',attrs={'valign':'middle'})
            #print len(vul)
            for j in range(0,len(vul)):
                url = "https://secuniaresearch.flexerasoftware.com"+vul[j].find('a')['href']

                yield Request(url, callback=self.save_vul)

    def save_vul(self,response):
        #print response.text
        self.currentVuls+=1.0
        schedule(self.totalVuls,self.currentVuls)
        try:
            item= SecuniaItem()
            vul = bs(response.text, 'lxml').find('pre').get_text()
            try:
                item['cveID'] = re.search(self.cvePattern, vul).group()
            except:
                item['cveID'] = None
            items = vul.split('======================================================================')

            item['url'] = response.url
            item['title'] = items[1].strip().splitlines()[-1].replace('-','').strip()
            content =""
            i=0
            for i in range(len(items)):
                if 'Table of Contents' in items[i]:
                    content =  items[i].strip()
            #        print content
                    break
            lines = content.splitlines()
            lineNum = len(lines)
            dic={}
            for j in range(0,lineNum):
                if 'Affected Software' in lines[j]:
                    affected =  items[i+j-1].strip().splitlines()
                    del affected[0]

                    item['effectSys'] = ''.join(affected)

                elif 'Severity' in lines[j]:
                    severity =items[i+j-1].strip().splitlines()
                    if 'Less critical' in severity[2] or 'Not critical' in severity[2]:
                        item['level'] = "低危"
                    elif 'Moderately critical' in severity[2]:
                        item['level'] = "中危"
                    elif 'Highly critical' in severity[2]:
                        item['level'] = '高危'
                    item['type'] = severity[3].split(': ')[-1]

                elif 'Description' in lines[j]:
                    description =items[i+j-1].strip().splitlines()
                    del description[0]
                    item['message'] = ''.join(description)
                elif 'Solution' in lines[j]:
                    solution =items[i+j-1].strip().splitlines()
                    del solution[0]
                    item['suggestion'] = ''.join(solution)
                elif 'Time' in lines[j]:
                    times =items[i+j-1]
                    times = re.findall(self.timePattern,times)
                    #print times
                    try:
                        item['updateTime'] = time.strftime("%Y-%m-%d", time.strptime(times[-1], "%Y/%m/%d"))
                    except:
                        item['updateTime'] = time.strftime("%Y-%m-%d", time.strptime(times[-1], "%d/%m/%Y"))
                    try:
                        item['publishTime'] =   time.strftime("%Y-%m-%d", time.strptime(times[0], "%Y/%m/%d"))
                    except:
                        item['publishTime'] = time.strftime("%Y-%m-%d", time.strptime(times[0], "%d/%m/%Y"))
            item['chinese'] = False
            item['current'] = int(self.currentVuls)
            item['total'] = self.totalVuls
            #print item
            yield item
            #print item
        except Exception as e:
            yield None

        #item['effectSys'] =
    #     #item['type'] = re.search(re.compile(r'MS\d{2}-\d+\s+-\s+(.*)'),vulbar.find('h1').get_text().strip()).group(1)
    #     #print item['type']
    #
    #     pattern_p = re.compile(r'Published:\s+([a-zA-Z]+\s+\d{1,}(,)\s\d{4})')
    #     pattern_u = re.compile(r'Updated:\s+([a-zA-Z]+\s+\d{1,}(,)\s\d{4})')
    #
    #     try:
    #         publishTime = response.xpath("//p[contains(text(),'Published')]/text()")[0].extract()
    #         publishTime = re.search(pattern_p, publishTime).group(1)
    #         item['publishTime'] = time.strftime("%Y-%m-%d", time.strptime(publishTime, "%B %d, %Y"))
    #
    #     except :
    #         item['publishTime'] = None
    #
    #
    #     try:
    #         updateTime = response.xpath("//p[contains(text(),'Updated')]/text()")[0].extract()
    #         updateTime = re.search(pattern_u, updateTime).group(1)
    #         item['updateTime']=time.strftime("%Y-%m-%d", time.strptime(updateTime, "%B %d, %Y"))
    #
    #     except :
    #         item['updateTime'] = None
    #
    #     #print item['updateTime']
    #
    #     pattern = re.compile(r'CVE-\d{4}-\d+')
    #     if re.search(pattern, vulbar.get_text()) is not None:
    #         item['cveID'] = re.search(pattern, vulbar.get_text()).group()
    #     else:
    #         item['cveID'] = None
    #     #print item['cveID']
    #     try:
    #         item['message'] = bs(response.text,'lxml').find(attrs={'id':'executive-summary'}).next_sibling.next_sibling.get_text()
    #     except:
    #         item['message'] = 'None'
    #     item['chinese'] = False
    #     item['total'] = self.totalVuls
    #     item['current'] = int(self.currentVuls)
    #     yield item
    #






